
#include "TxtLrn.h"

int main(int argc, const char* argv[])
{
	int nRetCode = 0;
	CMyRandom::InitRandomSeed();
	CImageEx::Initialize();

	if (1)
	{
		string strFN_Configuration		=	argv[1];
		string strOperation				=	argv[2];

		strOperation += "00000000";

		int op_SVM_FirstRound = CDataStructureTools::Hex2Num(strOperation[0]);
		int op_SVM_SecondRound = CDataStructureTools::Hex2Num(strOperation[1]);
		int op_KNN_FirstRound = CDataStructureTools::Hex2Num(strOperation[2]);
		int op_KNN_SecondRound = CDataStructureTools::Hex2Num(strOperation[3]);

		char strTemp[10001] = "";

		//////////	Config Loading	//////////

		CConfiguration Config(strFN_Configuration);
		Config.Load();

		string strFN_OutLog =
			Config.GetString("strFN_OutLog", "global.log");

		ofstream outLog(strFN_OutLog.c_str());
		CExternalTools::ConsoleTools::MakeCurrentLine("", true);
		CExternalTools::ConsoleTools::OutputTextToStream(outLog, "", true);

		EnvVar.strFN_Vocabulary =
			Config.GetString("strFN_Vocabulary");
		EnvVar.Vocabulary.LoadFromFile(EnvVar.strFN_Vocabulary);
		sprintf_s(strTemp, "Original Vocabulary Loaded;");
		CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
		CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

		EnvVar.IDF_List.Resize(EnvVar.Vocabulary.TermAmount, 0.0);

		EnvVar.strFN_Data_Train =
			Config.GetString("strFN_Data_Train");
		EnvVar.strFN_Data_Test =
			Config.GetString("strFN_Data_Test");
		EnvVar.strFN_Label_Train =
			Config.GetString("strFN_Label_Train");
		EnvVar.strFN_Label_Test =
			Config.GetString("strFN_Label_Test");

		EnvVar.strFN_LabelSettings_Train =
			Config.GetString("strFN_LabelSettings_Train");
		EnvVar.TargetBinaryLabelSets_Train.myInit(EnvVar.strFN_LabelSettings_Train);
		EnvVar.strFN_LabelSettings_Test =
			Config.GetString("strFN_LabelSettings_Test");
		EnvVar.TargetBinaryLabelSets_Test.myInit(EnvVar.strFN_LabelSettings_Test);

		EnvVar.Feature_DF_Threshold =
			Config.GetInt("Feature_DF_Threshold");
		if (EnvVar.Feature_DF_Threshold < 1)
			EnvVar.Feature_DF_Threshold = 1;

		EnvVar.strFN_SelectedVocabulary_1 =
			Config.GetString("strFN_SelectedVocabulary_1");
		EnvVar.SelectedVocabulary_1.InitVocabulary(&EnvVar.Vocabulary);
		EnvVar.strFN_SVM_TrainLog_1 =
			Config.GetString("strFN_SVM_TrainLog_1");
		EnvVar.strFN_SVM_Model_1 =
			Config.GetString("strFN_SVM_Model_1");
		EnvVar.strFN_SVM_ROC_1 =
			Config.GetString("strFN_SVM_ROC_1");

		EnvVar.SampleRate_1stRound_Pos_Train =
			Config.GetDouble("SampleRate_1stRound_Pos_Train", 1.00);
		EnvVar.SampleRate_1stRound_Neg_Train =
			Config.GetDouble("SampleRate_1stRound_Neg_Train", 1.00);

		EnvVar.strFN_SelectedVocabulary_2 =
			Config.GetString("strFN_SelectedVocabulary_2");
		EnvVar.SelectedVocabulary_2.InitVocabulary(&EnvVar.Vocabulary);
		EnvVar.strFN_SVM_TrainLog_2 =
			Config.GetString("strFN_SVM_TrainLog_2");
		EnvVar.strFN_SVM_Model_2 =
			Config.GetString("strFN_SVM_Model_2");
		EnvVar.strFN_SVM_ROC_2 =
			Config.GetString("strFN_SVM_ROC_2");

		EnvVar.strFN_ScoreList_Train =
			Config.GetString("strFN_ScoreList_Train");
		EnvVar.HardDataRatio =
			Config.GetDouble("HardDataRatio");
		EnvVar.SampleRate_EasyExample =
			Config.GetDouble("SampleRate_EasyExample");
		EnvVar.SampleRate_HardExample =
			Config.GetDouble("SampleRate_HardExample");

		EnvVar.KNN_MemoryCacheSize_InMB =
			Config.GetInt("KNN_MemoryCacheSize_InMB");

		EnvVar.strFN_IDF_Train_1 =
			Config.GetString("strFN_IDF_Train_1");
		EnvVar.KNN_K_1 =
			Config.GetInt("KNN_K_1");
		EnvVar.strFN_KNN_Model_1 =
			Config.GetString("strFN_KNN_Model_1");
		EnvVar.strFN_KNN_Data_1 =
			Config.GetString("strFN_KNN_Data_1");
		EnvVar.strFN_KNN_ROC_1 =
			Config.GetString("strFN_KNN_ROC_1");

		EnvVar.strFN_IDF_Train_2 =
			Config.GetString("strFN_IDF_Train_2");
		EnvVar.KNN_K_2 =
			Config.GetInt("KNN_K_2");
		EnvVar.strFN_KNN_Model_2 =
			Config.GetString("strFN_KNN_Model_2");
		EnvVar.strFN_KNN_Data_2 =
			Config.GetString("strFN_KNN_Data_2");
		EnvVar.strFN_KNN_ROC_2 =
			Config.GetString("strFN_KNN_ROC_2");

		const CVocabulary& Vocabulary = EnvVar.Vocabulary;
		//CSelectedVocabulary& SelectedVocabulary_1 = EnvVar.SelectedVocabulary_1;
		//CSelectedVocabulary& SelectedVocabulary_2 = EnvVar.SelectedVocabulary_2;
		const double* pIDF = EnvVar.IDF_List.pElement;

		// ********** Main Code Start **********
		if (1)
		{
			int i_model, i_round, i_command;

			CDocumentList DL_Train;
			CDataSetForBinaryClassification bds_Train;
			if (0
				|| (op_SVM_FirstRound & 5)
				|| (op_SVM_SecondRound & 1)
				|| (op_KNN_FirstRound & 5)
				|| (op_KNN_SecondRound & 1)
				)
			{
				DL_Train.InitVocabulary(&EnvVar.Vocabulary);
				DL_Train.InitDocumentList(EnvVar.strFN_Data_Train, EnvVar.strFN_Label_Train);
				sprintf_s(strTemp, "Original Train Data Loaded;");
				CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
				CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
				bds_Train.SetRealDataSet(&DL_Train);
			}

			CDocumentList DL_Test;
			CDataSetForBinaryClassification bds_Test;
			if (0
				|| (op_SVM_FirstRound & 2)
				|| (op_SVM_SecondRound & 2)
				|| (op_KNN_FirstRound & 2)
				|| (op_KNN_SecondRound & 2)
				)
			{
				DL_Test.InitVocabulary(&EnvVar.Vocabulary);
				DL_Test.InitDocumentList(EnvVar.strFN_Data_Test, EnvVar.strFN_Label_Test);
				sprintf_s(strTemp, "Original Test Data Loaded;");
				CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
				CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
				bds_Test.SetRealDataSet(&DL_Test);
			}

			for ( i_model = 0; i_model < 2; ++i_model )
			{
				int op_FirstRound, op_SecondRound;
				string strModelName;

				switch (i_model)
				{
				case 0:
					op_FirstRound = op_SVM_FirstRound;
					op_SecondRound = op_SVM_SecondRound;
					strModelName = "SVM";
					break;
				case 1:
					op_FirstRound = op_KNN_FirstRound;
					op_SecondRound = op_KNN_SecondRound;
					strModelName = "KNN";
					break;
				default:
					break;
				}

				for ( i_round = 1; i_round <= 2; ++i_round )
				{
					CSelectedVocabulary* pSelectedVocabulary;
					string strFN_SelectedVocabulary;
					string strFN_SVM_TrainLog;
					string strFN_SVM_Model;
					string strFN_SVM_ROC;
					string strFN_IDF_Train;
					int KNN_K;
					string strFN_KNN_Model;
					string strFN_KNN_Data;
					string strFN_KNN_ROC;

					switch (i_round)
					{
					case 1:
						pSelectedVocabulary = &EnvVar.SelectedVocabulary_1;
						strFN_SelectedVocabulary = EnvVar.strFN_SelectedVocabulary_1;
						strFN_SVM_TrainLog = EnvVar.strFN_SVM_TrainLog_1;
						strFN_SVM_Model = EnvVar.strFN_SVM_Model_1;
						strFN_SVM_ROC = EnvVar.strFN_SVM_ROC_1;
						strFN_IDF_Train = EnvVar.strFN_IDF_Train_1;
						KNN_K = EnvVar.KNN_K_1;
						strFN_KNN_Model = EnvVar.strFN_KNN_Model_1;
						strFN_KNN_Data = EnvVar.strFN_KNN_Data_1;
						strFN_KNN_ROC = EnvVar.strFN_KNN_ROC_1;
						break;
					case 2:
						pSelectedVocabulary = &EnvVar.SelectedVocabulary_2;
						strFN_SelectedVocabulary = EnvVar.strFN_SelectedVocabulary_2;
						strFN_SVM_TrainLog = EnvVar.strFN_SVM_TrainLog_2;
						strFN_SVM_Model = EnvVar.strFN_SVM_Model_2;
						strFN_SVM_ROC = EnvVar.strFN_SVM_ROC_2;
						strFN_IDF_Train = EnvVar.strFN_IDF_Train_2;
						KNN_K = EnvVar.KNN_K_2;
						strFN_KNN_Model = EnvVar.strFN_KNN_Model_2;
						strFN_KNN_Data = EnvVar.strFN_KNN_Data_2;
						strFN_KNN_ROC = EnvVar.strFN_KNN_ROC_2;
						break;
					default:
						break;
					}

					for ( i_command = 1; i_command <= 4; i_command += i_command )
					{
						if ((i_round==1) && (i_command==1) && (op_FirstRound&i_command))
						{
							sprintf_s(strTemp, "\nModel [%s], Round %d, Model Training:\n", strModelName.c_str(), i_round);
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

							bds_Train.CopyLabelSettingFrom(
								EnvVar.TargetBinaryLabelSets_Train.BinaryClassificationDataSetPrototype);
							CBinaryClassificationExampleList ExampleList_Train_1;
							CWeightedClassificationExampleList MixedList;
							{
								CBinaryClassificationExampleList tbList;
								tbList.myInit(&bds_Train);
								ExampleList_Train_1.myInit(tbList,
									EnvVar.SampleRate_1stRound_Pos_Train, EnvVar.SampleRate_1stRound_Neg_Train);
								sprintf_s(strTemp, "Set [%s] Document Sample Rate: Pos [%.2lf%%] / Neg [%.2lf%%];",
									EnvVar.GetText_ImageSet(0),
									EnvVar.SampleRate_1stRound_Pos_Train * 100.0,
									EnvVar.SampleRate_1stRound_Neg_Train * 100.0);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

								ExampleList_Train_1.ComputeTotalExampleWeight();
								CWeightedClassificationExampleList& PL = ExampleList_Train_1.PosList;
								CWeightedClassificationExampleList& NL = ExampleList_Train_1.NegList;
								int i, t;
								t = ExampleList_Train_1.GetTotalExampleAmount();
								for ( i = 0; i < PL.ExampleAmount; ++i )
									PL.rgExample[i].Weight /= PL.TotalExampleWeight / t;
								for ( i = 0; i < NL.ExampleAmount; ++i )
									NL.rgExample[i].Weight /= NL.TotalExampleWeight / t;
								ExampleList_Train_1.ComputeTotalExampleWeight();
							}

							// Feature Selection
							{
								ExampleList_Train_1.MakeMixedList(MixedList);
								DL_Train.DoFeatureSelection(
									*pSelectedVocabulary,EnvVar.IDF_List, EnvVar.Feature_DF_Threshold, MixedList);
								DL_Train.InitSelectedVocabulary(pSelectedVocabulary);
								pSelectedVocabulary->SaveToFile(strFN_SelectedVocabulary);
								sprintf_s(strTemp, "Selected Feature Dimension = %d;", (int)pSelectedVocabulary->SelectedAmount);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

							} // Feature Selection

							CSVMLightHelper::CParameters SVM_Parameters;
							CSVMLightHelper::CModel SVM_Model;
							CCosineKNN_Model::CParameters KNN_Config;
							CCosineKNN_Model KNN_Model;

							switch (i_model)
							{
							case 0:
								DL_Train.InitIDF(NULL);
								SVM_Parameters.SetFN_TrainLog(strFN_SVM_TrainLog);
								sprintf_s(strTemp, "-j %.6lf",
									1.0 * ExampleList_Train_1.NegList.TotalExampleWeight / ExampleList_Train_1.PosList.TotalExampleWeight);
								SVM_Parameters.myInit(strTemp);
								SVM_Model.Train(&bds_Train, ExampleList_Train_1, SVM_Parameters);
								SVM_Model.SaveToFile_F(strFN_SVM_Model, false);
								break;
							case 1:
								CDataStructureTools::SaveSimpleTypeArray(strFN_IDF_Train, EnvVar.IDF_List);
								DL_Train.InitIDF(EnvVar.IDF_List.pElement);
								KNN_Config.K = KNN_K;
								KNN_Config.strFN_TrainData = strFN_KNN_Data;
								KNN_Model.Train(&DL_Train, ExampleList_Train_1, KNN_Config,
									EnvVar.KNN_MemoryCacheSize_InMB, EnvVar.KNN_MemoryCacheSize_InMB);
								KNN_Model.SaveToFile(strFN_KNN_Model);
								break;
							default:
								break;
							}

							sprintf_s(strTemp, "Model Training Finished;");
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
						} // ((i_round==1) && (i_command==1) && (op_FirstRound&i_command))

						if ((i_round==1) && (i_command==4) && (op_FirstRound&i_command))
						{
							sprintf_s(strTemp, "\nModel [%s], Round %d, Train Set Score List Making:\n", strModelName.c_str(), i_round);
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

							bds_Train.CopyLabelSettingFrom(
								EnvVar.TargetBinaryLabelSets_Train.BinaryClassificationDataSetPrototype);
							CBinaryClassificationExampleList ExampleList_Train_Full;
							ExampleList_Train_Full.myInit(&bds_Train);

							pSelectedVocabulary->LoadFromFile(strFN_SelectedVocabulary);
							sprintf_s(strTemp, "Selected Feature Dimension = %d;", (int)pSelectedVocabulary->SelectedAmount);
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
							DL_Train.InitSelectedVocabulary(pSelectedVocabulary);

							CSVMLightHelper::CModel SVM_Model;
							CCosineKNN_Model KNN_Model;
							CBinaryClassifier* pBinaryClassifer;

							switch (i_model)
							{
							case 0:
								DL_Test.InitIDF(NULL);
								SVM_Model.LoadFromFile_F(strFN_SVM_Model, false);
								pBinaryClassifer = &SVM_Model;
								break;
							case 1:
								CDataStructureTools::LoadSimpleTypeArray(EnvVar.IDF_List, strFN_IDF_Train);
								DL_Train.InitIDF(EnvVar.IDF_List.pElement);
								KNN_Model.LoadFromFile(strFN_KNN_Model, EnvVar.KNN_MemoryCacheSize_InMB);
								pBinaryClassifer = &KNN_Model;
								break;
							default:
								break;
							}

							// Classify Train Set
							{
								CWeightedClassificationExampleList tList;
								int sgnPosNeg, i_example;
								double tw;

								ofstream outFile(EnvVar.strFN_ScoreList_Train.c_str());
								for ( sgnPosNeg = 0; sgnPosNeg < 2; ++sgnPosNeg )
								{
									if (sgnPosNeg==0)
										tList = ExampleList_Train_Full.NegList;
									else
										tList = ExampleList_Train_Full.PosList;
									for ( i_example = 0; i_example < tList.ExampleAmount; ++i_example )
									{
										tList.rgExample[i_example].Weight = pBinaryClassifer->Classify(
											&DL_Train, tList.rgExample[i_example].ExampleIndex);
									}
									sort(tList.rgExample.begin(), tList.rgExample.end(), &CWeightedClassificationExample::CompareByWeight);
									for ( i_example = 0; i_example < tList.ExampleAmount; ++i_example )
									{
										tw = tList.rgExample[i_example].Weight;
										sprintf_s(strTemp, "%d\t%+9.6lf", tList.rgExample[i_example].ExampleIndex, tw);
										outFile<<strTemp<<endl;

										if (CMyRandom::NextInt(tList.ExampleAmount/100)==0 || i_example+1==tList.ExampleAmount)
										{
											sprintf_s(strTemp, "Set [%s.%s] Document Making Score [%d/%d]...",
												EnvVar.GetText_ImageSet(0), EnvVar.GetText_PosNeg(sgnPosNeg),
												i_example+1, tList.ExampleAmount);
											CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, false);
										}
									}
									outFile<<"#"<<endl;
									sprintf_s(strTemp, "Set [%s.%s] Document Making Train Set Score [%d] Finished;",
										EnvVar.GetText_ImageSet(0), EnvVar.GetText_PosNeg(sgnPosNeg),
										tList.ExampleAmount);
									CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
									CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
								} // sgnPosNeg
								outFile.close();
								outFile.clear();
								sprintf_s(strTemp, "Set [%s] Making Score List Finished;",
									EnvVar.GetText_ImageSet(0));
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
							} // Classify Train Set

						} // ((i_round==1) && (i_command==4) && (op_FirstRound&i_command))

						if ((i_round==2) && (i_command==1) && (op_SecondRound&i_command))
						{
							sprintf_s(strTemp, "\nModel [%s], Round %d, Model Training:\n", strModelName.c_str(), i_round);
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

							// Load Score List
							CBinaryClassificationExampleList ExampleList_Easy_Train;
							CBinaryClassificationExampleList ExampleList_Hard_Train;
							double Threshold_EasyHardScore = 0.0;
							{
								ExampleList_Easy_Train.Clear();
								ExampleList_Hard_Train.Clear();

								CSimpleTypeArray<double> tList;
								tList.Clear();

								int sgnPosNeg, i_example;
								sgnPosNeg = i_example = 0;
								CBinaryClassificationExampleList ExampleList_Full_Train;
								ExampleList_Full_Train.Clear();
								CWeightedClassificationExample twe;
								double tw;

								ifstream inFile(EnvVar.strFN_ScoreList_Train.c_str());
								for ( sgnPosNeg = 0; sgnPosNeg < 2; ++sgnPosNeg )
								{
									while (inFile>>strTemp && strTemp[0]!='#')
									{
										twe.ExampleIndex = atoi(strTemp);
										inFile>>tw;
										twe.Weight = tw;

										if (sgnPosNeg==0)
										{
											ExampleList_Full_Train.NegList.PushBack(twe);
											tList.PushBack(-tw);
										}
										else
										{
											ExampleList_Full_Train.PosList.PushBack(twe);
											tList.PushBack(+tw);
										}
									}
								} // sgnPosNeg
								inFile.close();
								inFile.clear();

								sort(tList.Begin(), tList.End());
								i_example = (int)(tList.Size * EnvVar.HardDataRatio);
								if (i_example < 1)
									Threshold_EasyHardScore = tList.pElement[0] - EPS;
								else if (i_example >= (int)tList.Size)
									Threshold_EasyHardScore = tList.pElement[tList.Size-1] + EPS;
								else
									Threshold_EasyHardScore = 0.5 * (tList.pElement[i_example-1]+tList.pElement[i_example]);
								sprintf_s(strTemp, "Set [%s] Document Easy / Hard Threshold = %.6lf",
									EnvVar.GetText_ImageSet(0), Threshold_EasyHardScore);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

								for ( i_example = 0; i_example < ExampleList_Full_Train.NegList.ExampleAmount; ++i_example )
								{
									twe = ExampleList_Full_Train.NegList.rgExample[i_example];
									tw = twe.Weight; twe.Weight = 1.0;
									if (tw < -Threshold_EasyHardScore)
										ExampleList_Easy_Train.NegList.PushBack(twe);
									else
										ExampleList_Hard_Train.NegList.PushBack(twe);
								} // i_example
								sprintf_s(strTemp, "Set [%s.%s] Total Document: Easy [%d] / Hard [%d]",
									EnvVar.GetText_ImageSet(0), EnvVar.GetText_PosNeg(0),
									ExampleList_Easy_Train.NegList.ExampleAmount,
									ExampleList_Hard_Train.NegList.ExampleAmount);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
								for ( i_example = 0; i_example < ExampleList_Full_Train.PosList.ExampleAmount; ++i_example )
								{
									twe = ExampleList_Full_Train.PosList.rgExample[i_example];
									tw = twe.Weight; twe.Weight = 1.0;
									if (tw > +Threshold_EasyHardScore)
										ExampleList_Easy_Train.PosList.PushBack(twe);
									else
										ExampleList_Hard_Train.PosList.PushBack(twe);
								} // i_example
								sprintf_s(strTemp, "Set [%s.%s] Total Document: Easy [%d] / Hard [%d]",
									EnvVar.GetText_ImageSet(0), EnvVar.GetText_PosNeg(1),
									ExampleList_Easy_Train.PosList.ExampleAmount,
									ExampleList_Hard_Train.PosList.ExampleAmount);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
							} // Load Score List

							bds_Train.CopyLabelSettingFrom(
								EnvVar.TargetBinaryLabelSets_Train.BinaryClassificationDataSetPrototype);
							CBinaryClassificationExampleList ExampleList_Train_2;
							CWeightedClassificationExampleList MixedList;
							{
								sprintf_s(strTemp, "Set [%s.%s] Document Sample Rate: Easy [%.2lf%%] / Hard [%.2lf%%]",
									EnvVar.GetText_ImageSet(0), EnvVar.GetText_PosNeg(0),
									EnvVar.SampleRate_EasyExample * 100.0,
									EnvVar.SampleRate_HardExample * 100.0);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
								ExampleList_Train_2 = ExampleList_Easy_Train;
								ExampleList_Easy_Train.myInit(ExampleList_Train_2, EnvVar.SampleRate_EasyExample);
								ExampleList_Train_2 = ExampleList_Hard_Train;
								ExampleList_Hard_Train.myInit(ExampleList_Train_2, EnvVar.SampleRate_HardExample);
								sprintf_s(strTemp, "Set [%s.%s] Sampled Document: Easy [%d] / Hard [%d]",
									EnvVar.GetText_ImageSet(0), EnvVar.GetText_PosNeg(0),
									ExampleList_Easy_Train.NegList.ExampleAmount,
									ExampleList_Hard_Train.NegList.ExampleAmount);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
								sprintf_s(strTemp, "Set [%s.%s] Sampled Document: Easy [%d] / Hard [%d]",
									EnvVar.GetText_ImageSet(0), EnvVar.GetText_PosNeg(1),
									ExampleList_Easy_Train.PosList.ExampleAmount,
									ExampleList_Hard_Train.PosList.ExampleAmount);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
								ExampleList_Train_2.Clear();
								ExampleList_Easy_Train.MakeMixedList(ExampleList_Train_2.NegList);
								ExampleList_Hard_Train.MakeMixedList(ExampleList_Train_2.PosList);
								ExampleList_Train_2.MakeMixedList(MixedList);
								ExampleList_Train_2.myInit(&bds_Train, MixedList);

								ExampleList_Train_2.ComputeTotalExampleWeight();
								CWeightedClassificationExampleList& PL = ExampleList_Train_2.PosList;
								CWeightedClassificationExampleList& NL = ExampleList_Train_2.NegList;
								int i, t;
								t = ExampleList_Train_2.GetTotalExampleAmount();
								for ( i = 0; i < PL.ExampleAmount; ++i )
									PL.rgExample[i].Weight /= PL.TotalExampleWeight / t;
								for ( i = 0; i < NL.ExampleAmount; ++i )
									NL.rgExample[i].Weight /= NL.TotalExampleWeight / t;
								ExampleList_Train_2.ComputeTotalExampleWeight();
							}

							// Feature Selection
							{
								ExampleList_Train_2.MakeMixedList(MixedList);
								DL_Train.DoFeatureSelection(
									*pSelectedVocabulary,EnvVar.IDF_List, EnvVar.Feature_DF_Threshold, MixedList);
								DL_Train.InitSelectedVocabulary(pSelectedVocabulary);
								pSelectedVocabulary->SaveToFile(strFN_SelectedVocabulary);
								sprintf_s(strTemp, "Selected Feature Dimension = %d;", (int)pSelectedVocabulary->SelectedAmount);
								CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
								CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

							} // Feature Selection

							CSVMLightHelper::CParameters SVM_Parameters;
							CSVMLightHelper::CModel SVM_Model;
							CCosineKNN_Model::CParameters KNN_Config;
							CCosineKNN_Model KNN_Model;

							switch (i_model)
							{
							case 0:
								DL_Train.InitIDF(NULL);
								SVM_Parameters.SetFN_TrainLog(strFN_SVM_TrainLog);
								sprintf_s(strTemp, "-j %.6lf",
									1.0 * ExampleList_Train_2.NegList.TotalExampleWeight / ExampleList_Train_2.PosList.TotalExampleWeight);
								SVM_Parameters.myInit(strTemp);
								SVM_Model.Train(&bds_Train, ExampleList_Train_2, SVM_Parameters);
								SVM_Model.SaveToFile_F(strFN_SVM_Model, false);
								break;
							case 1:
								CDataStructureTools::SaveSimpleTypeArray(strFN_IDF_Train, EnvVar.IDF_List);
								DL_Train.InitIDF(EnvVar.IDF_List.pElement);
								KNN_Config.K = KNN_K;
								KNN_Config.strFN_TrainData = strFN_KNN_Data;
								KNN_Model.Train(&DL_Train, ExampleList_Train_2, KNN_Config,
									EnvVar.KNN_MemoryCacheSize_InMB, EnvVar.KNN_MemoryCacheSize_InMB);
								KNN_Model.SaveToFile(strFN_KNN_Model);
								break;
							default:
								break;
							}

							sprintf_s(strTemp, "Model Training Finished;");
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
						} // ((i_round==2) && (i_command==1) && (op_SecondRound&i_command))

						if (
							((i_round==1) && (i_command==2) && (op_FirstRound&i_command)) ||
							((i_round==2) && (i_command==2) && (op_SecondRound&i_command)) )
						{
							sprintf_s(strTemp, "\nModel [%s], Round %d, Model Testing:\n", strModelName.c_str(), i_round);
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

							bds_Test.CopyLabelSettingFrom(
								EnvVar.TargetBinaryLabelSets_Test.BinaryClassificationDataSetPrototype);
							CBinaryClassificationExampleList ExampleList_Test_Full;
							ExampleList_Test_Full.myInit(&bds_Test);

							pSelectedVocabulary->LoadFromFile(strFN_SelectedVocabulary);
							sprintf_s(strTemp, "Selected Feature Dimension = %d;", (int)pSelectedVocabulary->SelectedAmount);
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);
							DL_Test.InitSelectedVocabulary(pSelectedVocabulary);

							CSVMLightHelper::CModel SVM_Model;
							CCosineKNN_Model KNN_Model;
							CBinaryClassifier* pBinaryClassifer;
							string strFN_ROC;

							switch (i_model)
							{
							case 0:
								DL_Test.InitIDF(NULL);
								SVM_Model.LoadFromFile_F(strFN_SVM_Model, false);
								pBinaryClassifer = &SVM_Model;
								strFN_ROC = strFN_SVM_ROC;
								break;
							case 1:
								CDataStructureTools::LoadSimpleTypeArray(EnvVar.IDF_List, strFN_IDF_Train);
								DL_Test.InitIDF(EnvVar.IDF_List.pElement);
								KNN_Model.LoadFromFile(strFN_KNN_Model, EnvVar.KNN_MemoryCacheSize_InMB);
								pBinaryClassifer = &KNN_Model;
								strFN_ROC = strFN_KNN_ROC;
								break;
							default:
								break;
							}

							int sgnPosNeg, i_example, tP;
							double tC;
							CBinaryClassificationResult_ByConfidence bcrc;
							CBinaryClassificationResult_ByPrediction bcrp;
							bcrc.Clear();
							bcrp.Clear();

							for ( sgnPosNeg = 0; sgnPosNeg < 2; ++sgnPosNeg )
							{
								const CWeightedClassificationExampleList& tWEL = 
									(sgnPosNeg==0 ? ExampleList_Test_Full.NegList : ExampleList_Test_Full.PosList);
								for ( i_example = 0; i_example < tWEL.ExampleAmount; ++i_example )
								{
									pBinaryClassifer->Classify(&DL_Test, tWEL.rgExample[i_example].ExampleIndex, tC, tP);
									(sgnPosNeg==0 ? bcrc.Neg : bcrc.Pos).PushBack(tC);
									bcrp.NewTest((sgnPosNeg==0 ? -1 : +1), tP, 1.0);

									if (CMyRandom::NextInt(tWEL.ExampleAmount/100)==0 || i_example==tWEL.ExampleAmount-1)
									{
										sprintf_s(strTemp, "Set [%s.%s] Document Testing [%d/%d]...",
											EnvVar.GetText_ImageSet(1), EnvVar.GetText_PosNeg(sgnPosNeg),
											i_example+1, tWEL.ExampleAmount);
										CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, false);
									}
								}
							}
							bcrc.Analyse(); bcrp.Analyse();

							CExternalTools::ConsoleTools::MakeCurrentLine("", false);
							bcrp.OutputToStream_i(outLog);
							bcrp.OutputToStream_i(cout);

							sprintf_s(strTemp, "\nSet [%s] Testing Finished, Acc = %.2lf%%;",
								EnvVar.GetText_ImageSet(1), 100.0 - 100.0 * bcrp.ErrorRate);
							CExternalTools::ConsoleTools::MakeCurrentLine(strTemp, true);
							CExternalTools::ConsoleTools::OutputTextToStream(outLog, strTemp, true);

							ofstream outFile_ROC(strFN_ROC.c_str());
							bcrc.OutputROCtoStream(outFile_ROC, 100, 0.0, 1.0, false);
							outFile_ROC.close();
							outFile_ROC.clear();
						} // (	((i_round==1) && (i_command==2) && (op_FirstRound&i_command)) ||
						//		((i_round==2) && (i_command==2) && (op_SecondRound&i_command)) )
					} // i_command
				} // i_round
			} // i_model

		} // Main Code
			
		CExternalTools::ConsoleTools::MakeCurrentLine("", true);
		CExternalTools::ConsoleTools::OutputTextToStream(outLog, "", true);
		outLog.close();
		outLog.clear();
	}

	CImageEx::Release();
	return 0;
}

